import com.db4o.Db4o;
import com.db4o.ObjectContainer;
import com.sun.org.apache.xpath.internal.compiler.Keywords;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.url.WebURL;
import org.apache.commons.lang3.StringUtils;

import javax.naming.ldap.Control;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

public class Crawler extends WebCrawler {
    private ArrayList<Word> keywords;

	Pattern filters = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g"
			+ "|png|tiff?|mid|mp2|mp3|mp4" + "|wav|avi|mov|mpeg|ram|m4v|pdf"
			+ "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

	public Crawler() {
        keywords = new ArrayList<Word>();

        getWordList();
	}

	public boolean shouldVisit(WebURL url) {
		String href = url.getURL().toLowerCase();
		if (filters.matcher(href).matches()) {
			return false;
		}
		if (href.startsWith("http://www.cnn.com")
                || href.startsWith("http://www.bbc.co.uk")
                || href.startsWith("http://www.reuters.com")
                || href.startsWith("http://www.usatoday.com")
                || href.startsWith("http://www.bloomberg.com")
                || href.startsWith("http://www.google.com")
                || href.startsWith("http://news.google.com")) {
			return true;
		}
		return false;
	}
	
	public void visit(Page page) {
        String text = page.getText();
        String url = page.getWebURL().getURL();
		int pageRel = getRelevance(text);
        if (url.contains("sport") || url.contains("video")) {
            pageRel -= 1500;
        }
        System.out.println("I'M CHECKING: "+url);
        if (pageRel>999) {
            List<WebURL> links = page.getURLs();
		    int parentDocId = page.getWebURL().getParentDocid();
            int docId = page.getWebURL().getDocid();

            System.out.println();
            System.out.println("=====================");
            System.out.println("DocId: " + docId);
            System.out.println("URL: " + url);
            System.out.println("Text length: " + text.length());
            System.out.println("Number of links: " + links.size());
            System.out.println("DocId of parent page: " + parentDocId);
            System.out.println("Relevance: "+ pageRel);
            System.out.println("=====================");
            System.out.println();
            record(page);
        }
	}
    public void record(Page p) {
        try {
            Controller.db.store(p);
            Controller.db.commit();
        } catch (Exception e) {
            System.out.println("**************Record Error***********");
            e.printStackTrace();
        }
    }
    public void getWordList() {
          try {
              FileInputStream fstream = new FileInputStream("occupylist.txt");

              DataInputStream in = new DataInputStream(fstream);
              BufferedReader br = new BufferedReader(new InputStreamReader(in));
              String strLine;

              while ((strLine = br.readLine()) != null)   {
                  Word w = new Word(strLine);
                  keywords.add(w);
              }

              in.close();
          } catch (Exception e) {
              System.err.println("Error: " + e.getMessage());
          }
    }

    public int getRelevance(String text) {
        int rel = 0;
        for (Word word : keywords) {
            rel += word.getWeight() * StringUtils.countMatches(text.toLowerCase(), word.getWord().toLowerCase());
        }

        return rel;
    }



}